# -*- coding:utf-8 -*-
# --author： jingfeng
# time: 2019/1/23


import requests
from fake_useragent import UserAgent
from lxml import etree
import json
from bs4 import BeautifulSoup
import re
import csv

ua = UserAgent()
headers = {'User-Agent': ua.random}


def get_teacher_info():
    for i in range(1, 81):
        url = 'http://edu.51cto.com/center/lec/index/list?wzdh=&page=%s' % str(
            i)

        response = requests.get(url=url, headers=headers)
        response.encoding = response.apparent_encoding
        html = response.text
        selector = etree.HTML(html)

        myselector = selector.xpath('//ul[@class="Lecs"]/li')

        for selector in myselector:


            name = selector.xpath('./div[2]/h2/a/text()')[0]

            course_num_str = selector.xpath(
                './div[2]/div[@class="mt10 color81 ' 'mb10"]/p[1]/text()')[0]
            course_num = re.search(r'(\d+)', course_num_str)

            course_num = course_num.group()

            student_num_str = selector.xpath(
                './div[2]/div[@class' '="mt10 color81 mb10"]/p[2]/text()')[0]
            student_num = re.search(r'(\d+)', student_num_str)

            student_num = student_num.group()

            # 职称
            title = selector.xpath('./div[2]/p[1]/text()')[0]

            # 教师信息
            teacher_info = selector.xpath('./div[2]/p[2]/text()')
            teacher_info = ','.join(teacher_info)

            course_link = selector.xpath('./div[2]/a[1]/@href')[0]

            blog_link = selector.xpath('./div[2]/a[2]/@href')
            if blog_link:
                blog_link = blog_link[0]
            else:
                blog_link = '没有博客'

            # 最新课程
            new_course = selector.xpath('./div[@class="courseList"]/a/text()')
            new_course = ','.join(new_course)

            list = [name, course_num, student_num, title, teacher_info,
                    course_link, blog_link, new_course]

            # print(name)
            # print(course_num)
            # print(student_num)
            # print(title)
            # print(teacher_info)
            # print(course_link)
            # print(blog_link)
            # print(new_course)

            save_info(list)

        print('第-%s-页完成' % str(i))


file = open('teacher.csv', 'a', encoding='utf-8', newline='')
writer = csv.writer(file)

writer.writerow(['name', 'course_num', 'student_num', 'title', 'teacher_info',
                 'course_link', 'blog_link', 'new_course'])


def save_info(info):
    writer.writerow(info)


get_teacher_info()
